﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Markup;
using System.Xaml;

using alib;
using alib.Bits;
using alib.Collections;
using alib.Collections.ReadOnly;
using alib.Enumerable;

using agree.schema;

namespace agree
{
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// 
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[ContentProperty("Item")]
	public class TokLoader : m_observable<IString, TokenSpanToken>,
		ρρTokenizer,
		ρρTokenSet
	{
		public TokLoader(IRtParent parent, IString item)
			: base(parent, item)
		{
			this.itok = SplitTokenizer.SpaceCharTokenizer;
			this.ts_input = new TokenizedString(this, item.Text, itok);	/// error error
		}

		///////////////////////////////////////////////////////////////////////
		///
		public IString Item
		{
			get { return t0; }
			set { t0 = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		///
		ITokenizer itok;
		public ITokenizer Tokenizer
		{
			get { return this.itok; }
			set { this.itok = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		///
		TokenSet ts_input;
		public TokenSet TokenSet
		{
			get { return this.ts_input; }
			set { this.ts_input = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		protected override void start(IString _ignored)
		{
			foreach (var tok in ts_input)
				_item(tok);

			_remove_task();
		}
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// ITokenizers return their results as a set of TokenizerTokens
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[DebuggerDisplay("{ToString(),nq}")]
	public sealed class CharacterSpanToken : identity_base, IEquatable<CharacterSpanToken>, ICharacterSpan
	{
		public static TokenizedString THE_TOKENIZED_STRING;

		public CharacterSpanToken(IIdentity prv, ChartSpan span, String modified)
			: base(prv)
		{
			if (modified != null && modified.Length != span.Length)
				Debug.WriteLine("Warning: The replacement text {0} contains a different number of characters than the span it replaces in the source {1}.", modified, span);
			this.span = span;
			this.modified = modified;
		}

		public CharacterSpanToken(IIdentity prv, ChartSpan span)
			: this(prv, span, null)
		{
		}

		public CharacterSpanToken(IIdentity prv, int i_start, int i_end)
			: this(prv, new ChartSpan(i_start, i_end))
		{
		}

		public CharacterSpanToken(IIdentity prv, int i_start, int i_end, String modified)
			: this(prv, new ChartSpan(i_start, i_end), modified)
		{
		}

		/// <summary> Character span in the source text </summary>
		ChartSpan span;
		public ChartSpan CharacterSpan { get { return span; } }

		///////////////////////////////////////////////////////////////////////
		/// <summary> A tokenizer can return replacement text which will be 
		/// associated with the parse token. If this is null, the text is 
		/// retrieved according to the span in the source text. </summary>
		/// 
		[DebuggerBrowsable(DebuggerBrowsableState.Never)]
		String modified;
		public String Text
		{
			get { return modified; }
			set { throw not.valid; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		/// <summary>
		/// helper function converts an array of strings to a sequence of abutting character spans.
		/// </summary>
		public static ICharacterSpan[] GetStringArraySpans(IIdentity prv, IEnumerable<String> word_array, int cch_sep)
		{
			ICharacterSpan[] arr = new ICharacterSpan[word_array._Count()];
			int i = 0, j = 0;
			foreach (String w in word_array)
			{
				if (w.Length > 0)
					arr[j++] = new CharacterSpanToken(prv, i, i + w.Length - 1);
				i += w.Length + cch_sep;
			}
			if (j < arr.Length)
				Array.Resize<ICharacterSpan>(ref arr, j);
			return arr;
		}

		public bool Equals(CharacterSpanToken other)
		{
			return span.Equals(other.span) && modified == other.modified;
		}

		public override bool Equals(object obj)
		{
			throw new Exception("--> use strongly typed IEquatable<T> instead");
		}

		public override int GetHashCode()
		{
			return (modified == null ? 0 : modified.GetHashCode()) ^ span.GetHashCode();
		}

		public override string ToString()
		{
			return String.Format("{0} [{1}]", span, modified);
		}
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// 
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[DebuggerDisplay("{ToString(),nq}")]
	public class TokenSpanToken : identity_base, ITokenSpan, IEquatable<TokenSpanToken>, IEnumerable<TokenSpanToken>,
		ρρTokenSpan
	{
		public TokenSpanToken(TokenSet ts, String s_subst, ChartSpan chr_span, ChartSpan tok_span)
			: base(ts)
		{
			this.ts = ts;
			this.s_subst = s_subst;
			this.chr_span = chr_span;
			this.tok_span = tok_span;
		}
		public TokenSpanToken(TokenSet ts, ICharacterSpan ch, ChartSpan tok_span)
			: this(ts, ch.Text, ch.CharacterSpan, tok_span)
		{
		}
		TokenSet ts;
		ChartSpan chr_span;
		ChartSpan tok_span;
		String s_subst;

		/// <summary> The span of characters in the source string that is covered by this token </summary>
		public ChartSpan CharacterSpan { get { return chr_span; } }

		/// <summary> The span of minimal token spans (e.g. chart positions) that is covered by this token </summary>
		public ChartSpan TokenSpan
		{
			get { return tok_span; }
			set { throw not.valid; }
		}

		///////////////////////////////////////////////////////////////////////
		/// <summary> The text of this token </summary>
		/// 
		public String Text
		{
			get
			{
				if (s_subst != null)
					return s_subst;
				TokenizedString toks = this.ts as TokenizedString;
				if (toks != null)
					return toks.Text.Substring(chr_span.StartIndex, chr_span.Length);

				return String.Empty;
			}
			set { throw not.valid; }
		}
		/// 
		///////////////////////////////////////////////////////////////////////

		/// <summary>
		/// True if this is an initial token in the source string.
		/// </summary>
		public bool IsFirst { get { return this.tok_span.StartIndex == 0; } }

		/// <summary>
		/// True if this is a final token in the source string.
		/// </summary>
		public bool IsLast { get { return this.tok_span.EndIndex == ts.TokenLimit; } }

		/// <summary>
		/// Enumerates all tokens in this token's TokenizedString which are strictly right-adjacent to this token.
		/// </summary>
		public IEnumerable<TokenSpanToken> RightAbuttingTokens()
		{
			int ix = this.tok_span.EndIndex + 1;
			foreach (TokenSpanToken tok in ts)
				if (tok.TokenSpan.StartIndex == ix)
					yield return tok;
		}

		/// <summary>
		/// Enumerates all possible sequences of strictly adjacent tokens to the right of this token
		/// </summary>
		public IEnumerable<IEnumerable<TokenSpanToken>> RightAdjacencySequences()
		{
			foreach (TokenSpanToken tok in RightAbuttingTokens())
			{
				bool f_any = false;
				foreach (IEnumerable<TokenSpanToken> adj in tok.RightAdjacencySequences())
				{
					yield return adj.Prepend(tok);
					f_any = true;
				}
				if (!f_any)
					yield return tok;
			}
		}

		/// <summary>
		/// Enumerates all tokens in this token's TokenizedString which are strictly left-adjacent to this token.
		/// </summary>
		public IEnumerable<TokenSpanToken> LeftAbuttingTokens()
		{
			int ix = this.tok_span.StartIndex - 1;
			foreach (TokenSpanToken tok in ts)
				if (tok.TokenSpan.EndIndex == ix)
					yield return tok;
		}

		/// <summary>
		/// Enumerates all possible sequences of strictly adjacent tokens to the left of this token
		/// </summary>
		public IEnumerable<IEnumerable<TokenSpanToken>> LeftAdjacencySequences()
		{
			foreach (TokenSpanToken tok in LeftAbuttingTokens())
			{
				bool f_any = false;
				foreach (IEnumerable<TokenSpanToken> adj in tok.LeftAdjacencySequences())
				{
					yield return adj.Append(tok);
					f_any = true;
				}
				if (!f_any)
					yield return tok;
			}
		}

		public bool Equals(TokenSpanToken other)
		{
			return this.ts == other.ts && this.chr_span.Equals(other.chr_span) && this.tok_span.Equals(other.tok_span);
		}

		public override bool Equals(object obj)
		{
			return obj is TokenSpanToken && this.Equals((TokenSpanToken)obj);
		}

		public override int GetHashCode()
		{
			return ts.GetHashCode() ^ (chr_span.GetHashCode() - tok_span.GetHashCode());
		}

		IEnumerator<TokenSpanToken> IEnumerable<TokenSpanToken>.GetEnumerator()
		{
			return new UnaryCollection<TokenSpanToken>._enum(this);
		}

		IEnumerator IEnumerable.GetEnumerator()
		{
			return new UnaryCollection<TokenSpanToken>._enum(this);
		}

		public override String ToString()
		{
			return String.Format("{0,2} {1,7} {2,7} {3} {4} {5}",
				ts.IndexOf(this),
				chr_span,
				tok_span,
				Text,
				IsFirst ? "First" : "",
				IsLast ? "Last" : "");
		}
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// Abstract base class for TokenizedString, which represents a string and its arbitrary collection of tokens,
	/// and TokenizationHypothesis, which represents a covering tokenization solution.
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public abstract class TokenSet : IdentList<TokenSpanToken>
	{
		/// <summary>
		/// bypass the minimal coverage calculation and thus allow recording token selections for a covering subset
		/// </summary>
		//protected TokenSet(IEnumerable<TokenSpanToken> rg_tok)
		//    : base(rg_tok)
		//{
		//}

		/// <summary>
		/// Construct a derived object for the given string and array of character spans. This constructor is
		/// intended for internal use.
		/// </summary>
		protected unsafe TokenSet(IIdentity prv, ICharacterSpan[] rg_tok)
			: base(prv)
		{
			/// each character span designates a token. Any number of tokens may overlap or leave gaps in the string 
			/// in an arbitrary fashion. Uncovered gaps are ignored by the minimal token span that is computed, making
			/// them appropriate for representing whitespace.
			if (rg_tok.Length == 0)
				return;

			var mts = minimal_token_spans(rg_tok);

			/// create tokens with source character span and remapped minimal token span
			for (int i = 0; i < rg_tok.Length; i++)
				base.Add(new TokenSpanToken(this, rg_tok[i], mts[i]));
		}

		public abstract bool HasExclusion { get; }

		static unsafe bool get_map<T>(IReadOnlyList<T> toks, int* map) where T : ICharacterSpan
		{
			ChartSpan cs;
			bool f_simple = true;
			for (int i = 0, c = toks.Count, cur, prev = -1; i < c; i++)
			{
				cs = toks[i].CharacterSpan;
				Debug.Assert(cs.IsValid);
				map[cur = cs.StartIndex] |= 1;
				f_simple &= (cur > prev);
				prev = cs.EndIndex;
				while (cur <= prev)
					map[cur++] |= 2;
				map[cur] |= 1;
			}
			return f_simple;
		}
		static unsafe bool get_map(IReadOnlyList<ISourceWord> toks, int* map)
		{
			ChartSpan cs;
			bool f_simple = true;
			for (int i = 0, c = toks.Count, cur, prev = -1; i < c; i++)
			{
				cs = new ChartSpan(toks[i]);
				Debug.Assert(cs.IsValid);
				map[cur = cs.StartIndex] |= 1;
				f_simple &= (cur > prev);
				prev = cs.EndIndex;
				while (cur <= prev)
					map[cur++] |= 2;
				map[cur] |= 1;
			}
			return f_simple;
		}

		public static unsafe ChartSpan[] minimal_token_spans(ICharacterSpan[] toks)
		{
			int i, j, cur, c;
			if ((c = toks.Length) == 0)
				return Collection<ChartSpan>.None;

			int cch = covered_char_limit(toks) + 2;			/// +1 for index-to-count, +1 for EndIndex+1
			int* map = stackalloc int[cch];
			bool f_simple = get_map(toks, map);

			ChartSpan[] rg = new ChartSpan[c];
			if (!f_simple)
			{
				cch--;
				for (i = 0, cur = -1; i < cch; i++)
					if (((j = map[i]) & 2) == 0)
						map[i] = -1;
					else
					{
						if ((j & 1) != 0)
							cur++;
						map[i] = cur;
					}

				ChartSpan cs;
				for (i = 0; i < c; i++)
					rg[i] = new ChartSpan(map[(cs = toks[i].CharacterSpan).StartIndex], map[cs.EndIndex]);
			}
			else
				for (i = 0; i < c; i++)
					rg[i] = new ChartSpan(i, i);

			return rg;
		}

		public static unsafe ChartSpan[] minimal_token_spans(IReadOnlyList<ISourceWord> toks, out int limit)
		{
			if (0 == (limit = toks.Count))
				return Collection<ChartSpan>.None;

			int cur, i, j, k = covered_char_limit(toks) + 1;			/// +1 for index-to-count
			int* map = stackalloc int[k];
			bool f_simple = get_map(toks, map);

			ChartSpan[] rg = new ChartSpan[limit];
			if (!f_simple)
			{
				k--;
				for (i = 0, cur = -1; i < k; i++)
					if (((j = map[i]) & 2) == 0)
						map[i] = -1;
					else
					{
						if ((j & 1) != 0)
							cur++;
						map[i] = cur;
					}

				limit = 0;
				for (i = 0; i < rg.Length; i++)
				{
					if ((k = map[(int)P.SourceCharacterEnd.GetValue(toks[i]) - 1]) > limit)
						limit = k;
					rg[i] = new ChartSpan(map[(int)P.SourceCharacterStart.GetValue(toks[i])], k);
				}
			}
			else
				for (i = 0; i < limit; i++)
					rg[i] = new ChartSpan(i, i);

			return rg;
		}

		/// <summary>
		/// The minimal set of non-overlapping (and possibly disjoint) character spans which accomodate 
		/// the reported source tokens **excluding gaps**
		/// calculated in constructor now and saved with the tokens
		/// This function is not intended for main-line use, as it reconstructs the analysis which is now
		/// performed in the constructor
		/// </summary>
		public unsafe ChartSpan[] MinimalCharacterSpans
		{
			get
			{
				int i, j, prev, c;
				if ((c = this.Count) == 0)
					return Collection<ChartSpan>.None;

				int cch = covered_char_limit(this) + 2;	/// +1 for index-to-count, +1 for EndIndex+1
				int* map = stackalloc int[cch];
				bool f_simple = get_map(this, map);

				ChartSpan[] rg;
				if (!f_simple)
				{
					rg = new ChartSpan[cch];
					for (i = 0, prev = -1, c = 0; i < cch; i++)
						if (((j = map[i]) & 2) == 0)
						{
							if (prev != -1)
								rg[c++] = new ChartSpan(prev, i - 1);
							prev = -1;
						}
						else if ((j & 1) != 0)
						{
							if (prev != -1)
								rg[c++] = new ChartSpan(prev, i - 1);
							prev = i;
						}

					if (c < cch)
						Array.Resize<ChartSpan>(ref rg, c);
				}
				else
				{
					rg = new ChartSpan[c];
					for (i = 0; i < c; i++)
						rg[i] = this[i].CharacterSpan;
				}
				return rg;
			}
		}

		/// <summary>
		/// The minimal set of non-overlapping (and possibly disjoint) character spans which accomodate 
		/// the reported tokens **plus gaps between them**
		/// This function is not intended for main-line use, as it reconstructs the analysis which is now
		/// performed in the constructor
		/// </summary>
		public unsafe ChartSpan[] CharacterBoundarySpans
		{
			get
			{
				int i, prev, c;
				int cch = CharacterLimit + 2;	/// +1 for index-to-count, +1 for EndIndex+1
				int* map = stackalloc int[cch];
				get_map(this, map);

				ChartSpan[] rg = new ChartSpan[cch];
				cch--;
				for (i = 1, prev = 0, c = 0; i <= cch; i++)
					if ((map[i] & 1) != 0)
					{
						rg[c++] = new ChartSpan(prev, i - 1);
						prev = i;
					}

				i--;
				if (prev < i)
					rg[c++] = new ChartSpan(prev, i - 1);

				if (c < cch)
					Array.Resize<ChartSpan>(ref rg, c);
				return rg;
			}
		}

		/// <summary>
		/// Returns true if any tokens overlap each other
		/// </summary>
		public bool AnyOverlaps
		{
			get
			{
				int i = base.Count - 1;
				while (i > 0)
					if (this[i].CharacterSpan.Overlaps(this[--i].CharacterSpan))
						return true;
				return false;
			}
		}

		public int CharacterLimit
		{
			get
			{
				TokenizedString tts = this as TokenizedString;
				int i = tts == null ? 0 : tts.Text.Length - 1;
				int j = Count == 0 ? 0 : covered_char_limit(this);
				return j > i ? j : i;
			}
		}

		public int CoveredCharacterLimit { get { return covered_char_limit(this); } }
		static int covered_char_limit<T>(IReadOnlyList<T> rg) where T : ICharacterSpan
		{
			int i, j, i_max;
			if ((i = rg.Count) <= 0)
				throw new InvalidOperationException();
			i_max = 0;
			while (--i >= 0)
				if ((j = rg[i].CharacterSpan.EndIndex) > i_max)
					i_max = j;
			return i_max;
		}
		static int covered_char_limit(IReadOnlyList<ISourceWord> rg)
		{
			int i, j, i_max;
			if ((i = rg.Count) <= 0)
				throw new InvalidOperationException();
			i_max = 0;
			while (--i >= 0)
			{
				j = (int)P.SourceCharacterEnd.GetValue(rg[i]);
				if (j > i_max)
					i_max = j;
			}
			return i_max;
		}

		/// <summary>
		/// Highest span index. Add one for the number of chart positions needed.
		/// </summary>
		public int TokenLimit
		{
			get
			{
				int i, j, i_max;
				if ((i = this.Count) <= 0)
					throw new InvalidOperationException();
				i_max = 0;
				while (--i >= 0)
					if ((j = this[i].TokenSpan.EndIndex) > i_max)
						i_max = j;
				return i_max;
			}
		}

		/// <summary>
		/// The set union of all character indexes (in the source string) which are referenced by the tokens
		/// </summary>
		public unsafe int[] UsedCharacterIndexes
		{
			get
			{
				int i, j;
				if ((i = Count) == 0)
					return Collection<int>.None;

				int cul = covered_char_limit(this) + 1;
				ulong* pul = stackalloc ulong[cul = ((cul - 1) >> 6) + 1];
				BitHelper bh = new BitHelper(pul, cul);

				ChartSpan sp;
				while (--i >= 0)
					for (j = (sp = this[i].CharacterSpan).StartIndex; j <= sp.EndIndex; j++)
						bh.SetBit(j);
				return bh.Positions;
			}
		}

#if false
		/// <summary>
		/// The set union of all token indexes (in the minimal token spans) which are referenced by the
		/// tokens. By the construction of 'TokenizationString' objects, this set should always be 
		/// uninterestingly equal to the range of integers from 0...MinimalSpanCount-1.
		/// For other objects which may have selected a subset of those tokens, however
		/// the range may have gaps which represent missing coverage vis-a-vis the originating
		/// 'TokenizationString'
		/// </summary>

		// gcs : 5/2012 - this is the same as Enumerable.Range<int>(0,this.Count-1);
		public unsafe int[] UsedTokenIndexes
		{
			get
			{
				int cul = ((MinimalSpanCount - 1) >> 6) + 1;
				ulong* pul = stackalloc ulong[cul];
				BitHelper bh = new BitHelper(pul, cul);

				var e = this.GetEnumerator();
				while (e.MoveNext())
				{
					ChartSpan sp = e.Current.TokenSpan;
					for (int i = sp.StartIndex; i <= sp.EndIndex; i++)
						bh.MarkBit(i);
				}
				return bh.Positions;
			}
		}
#endif
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// A string which carries information about multiple tokenization hypotheses. Tokens can overlap, and/or
	/// there can be gaps in the coverage of the surface string.
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public class TokenizedString : TokenSet, ρρText
	{
		readonly String m_string;

		static ICharacterSpan[] _prepare_char_spans(IIdentity prv, String s_input, IEnumerable<ICharacterSpan> iecs)
		{
			//if (iecs is ICollection)
			//    alib.Debugging.Nop.CodeCoverage("can optimize");

			int cch = s_input.Length;
			HashSet<CharacterSpanToken> hs = new HashSet<CharacterSpanToken>();
			foreach (var tt in iecs)
			{
				ChartSpan cs = tt.CharacterSpan;
				if (cs.EndIndex < cs.StartIndex || cs.StartIndex < 0 || cs.EndIndex >= cch)
					throw new IndexOutOfRangeException("Token character index is not within the range of the input string.");
				var cst = tt.Text == null || !(tt is CharacterSpanToken) ?
						new CharacterSpanToken(prv, cs, tt.Text ?? s_input.Substring(cs.StartIndex, cs.Length)) :
						(CharacterSpanToken)tt;
				hs.Add(cst);
			}
			CharacterSpanToken[] rgcst = new CharacterSpanToken[hs.Count];
			hs.CopyTo(rgcst);
			//Array.Sort(rgcst, new Comparison<CharacterSpanToken>((x, y) =>
			//    {
			//        int d = x.CharacterSpan.StartIndex - y.CharacterSpan.StartIndex;
			//        if (d != 0)
			//            return d;
			//        return x.CharacterSpan.EndIndex - y.CharacterSpan.EndIndex;
			//    }));
			return rgcst;
		}

		public TokenizedString(IIdentity prv, String s_input, IEnumerable<ICharacterSpan> iecs)
			: base(prv, _prepare_char_spans(prv, s_input, iecs))
		{
			/// The source string is never modified from its original form but individual tokens can manifest
			/// substitution text
			m_string = s_input;
		}

		/// <summary>
		/// Initialize a TokenizedString object by tokenizing the given string with the specified tokenizer
		/// </summary>
		public TokenizedString(IIdentity prv, String s_input, ITokenizer itok)
			: this(prv, s_input, itok.Tokenize(s_input))
		{
		}

		/// <summary>
		/// Initialize a TokenizedString object from an array of strings. The tokens are accepted as-is, but
		/// the surface representation incorporates the specified string as a seprarator between each string.
		/// </summary>
		public TokenizedString(IIdentity prv, IEnumerable<String> word_array, String separator = "")
			: base(prv, CharacterSpanToken.GetStringArraySpans(prv, word_array, separator.Length))
		{
			m_string = String.Join(separator, word_array);
		}

		/// <summary>
		/// For when the ISpanToken.Text is available, the source string is built from them
		/// </summary>
		/// <param name="i_span_toks"></param>
		public TokenizedString(IIdentity trace, ICharacterSpan[] i_span_toks, String separator = null)
			: base(trace, i_span_toks)
		{
			m_string = String.Join(separator, i_span_toks.Select(ist => ist.Text));
		}

		/// <summary>
		/// The original, unmodified text of the source string.
		/// </summary>
		public String Text
		{
			get { return m_string; }
			set { throw not.valid; }
		}

		/// <summary>
		/// Returns true if the tokens leave any uncovered gaps in the source string. 
		/// All objects of type 'TokenizedString' have fully covering token spans by construction, because the
		/// total coverage of the tokens that are incorporated when the object is built is how the contentful
		/// regions are permanently defined. Subsets of these tokens--which can manifest partial coverage in 
		/// relation to an originating 'TokenizedString'--are represented with the 'TokenizationHypothesis' 
		/// sibling object.
		/// </summary>
		public override bool HasExclusion { get { return UsedCharacterIndexes.Length != Text.Length; } }

		/// <summary>
		/// Permits access to the text of an arbitrary span in the source string. For one thing, this allows one 
		/// to examine the contents of the gap portions, if desired.
		/// </summary>
		public String SpanText(ChartSpan sp)
		{
			return m_string.Substring(sp.StartIndex, sp.Length);
		}

		/// <summary>
		/// This function is not intended for main-line use, as it reconstructs the analysis which is now
		/// performed in the constructor
		/// </summary>
		public ChartSpan MinimalSpanToCharacterSpan(ChartSpan sp_min)
		{
			var mcs = MinimalCharacterSpans;
			return new ChartSpan(mcs.ElementAt(sp_min.StartIndex).StartIndex,
							mcs.ElementAt(sp_min.EndIndex).EndIndex);
		}

		public ChartSpan CharacterSpanToMinimalSpan(ChartSpan sp)
		{
			var mcs = MinimalCharacterSpans;
			int i_start = -1, i_end = -1;
			for (int i = 0; i < mcs.Length; i++)
			{
				if (i_start == -1 && sp.StartIndex <= mcs[i].EndIndex)
					i_start = i;

				if (i_start != -1 && sp.EndIndex <= mcs[i].EndIndex)
				{
					i_end = i;
					break;
				}
			}

			if (i_end == -1)
				throw new Exception();

			return new ChartSpan(i_start, i_end);
		}

		public String MinimalSpanText(ChartSpan sp_min)
		{
			ChartSpan sp_chr = MinimalSpanToCharacterSpan(sp_min);
			return m_string.Substring(sp_chr.StartIndex, sp_chr.Length);
		}
#if fase
		/// <summary>
		/// Returns those token sequences which fully cover the minimal token spans, if any, as individual single-
		/// coverage hypotheses. Naturally some of these distinct sequences may re-use the same tokens; what this 
		/// function tries to guarantee is that each returned solution has no overlap. If one wishes to allow the 
		/// parser (for example) to take control of token selection, then one can just process the TokenizedString 
		/// to the parser instead, since such tools are expected to rely only on the common base class 'TokenSet'
		/// </summary>
		public IEnumerable<TokenizationHypothesis> Hypotheses
		{
			get
			{
				if (tokens.Length == 0)
					yield break;
				int ix_last = MinimalSpanCount - 1;
				foreach (var s in tokens[0].RightAdjacencySequences().Where(seq => seq.Last().TokenSpan.EndIndex == ix_last))
					yield return new TokenizationHypothesis(this, s);
			}
		}
#endif
		public override string ToString()
		{
			return this.Select(tok => String.Format("{{{0}}}{1}", tok.Text, tok.CharacterSpan.ToString())).StringJoin(" ");
		}
	};
#if false
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// As discussed above, TokenizedString has the ability to winnow all single coverage tokenization solutions from 
	/// a set of arbitrarily covering tokens (if any). These solutions are kept in this separate wrapper in order to 
	/// record their provenance.
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public class TokenizationHypothesis : TokenSet
	{
		TokenizedString ts;

		public TokenizationHypothesis(TokenizedString ts, IEnumerable<TokenSpanToken> tokens)
			: base(tokens)
		{
			this.ts = ts;
		}

		public override TokenizedString Source
		{
			get { return ts; }
		}

		/// <summary>
		/// Returns true if this hypothesis fully covers all minimal span positions of the tokenization
		/// superset (i.e. 'TokenizedString') from which this TokenizationHypothesis was drawn.
		/// </summary>
		public override bool HasExclusion
		{
			get
			{
				return this.UsedCharacterIndexes.Length != ts.UsedCharacterIndexes.Count;
			}
		}
	};
#endif

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// A simple tokenizer which splits text at the position of every instance of the (specified) characters
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[DisplayName("Split Tokenizer")]
	[Description("A simple tokenizer which can split strings according to a configurable set of one or more delimiting characters. " +
				"The first delimiter in the list is used as the separator when joining words to make a surface string.")]
	public class SplitTokenizer : identity_base, ITokenizer,
		ρρDefaultWordBreak
	{
		public static readonly ITokenizer SpaceCharTokenizer = new SplitTokenizer(SysObj.Instance);

		public SplitTokenizer(IIdentity prv)
			: base(prv)
		{
			this.sep_chars = alib.Character.Charset.sp;
			this.def_sep = " ";
		}

		Char[] sep_chars;
		public Char[] SeparatorCharacters
		{
			get { return this.sep_chars; }
			set { this.sep_chars = value; }
		}

		String def_sep;
		public String DefaultWordBreak
		{
			get { return def_sep; }
			set { def_sep = value; }
		}

		public IEnumerable<ICharacterSpan> Tokenize(string s_input)
		{
			int ix, ix_prev = 0;
			while ((ix = s_input.IndexOfAny(sep_chars, ix_prev)) != -1)
			{
				if (ix > ix_prev)
					yield return new CharacterSpanToken(this, ix_prev, ix - 1);
				ix_prev = ix + 1;
			}
			if (s_input.Length > ix_prev)
				yield return new CharacterSpanToken(this, ix_prev, s_input.Length - 1);
		}

		public String MakeSurface(IEnumerable<String> words)
		{
			bool multi;
			using (var e = words.GetEnumerator())
				multi = e.MoveNext() && e.MoveNext();

			StringBuilder sb = new StringBuilder();
			int i = 0;
			using (var e = words.GetEnumerator())
				while (e.MoveNext())
				{
					String w = e.Current;
					if (multi || (w.Length == 1 && w[0] == 'i'))
					{
						w = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(w);
						multi = false;
					}
					if (i > 0)
						sb.Append(DefaultWordBreak);
					sb.Append(w);
					i++;
				}
			return sb.ToString();
		}
	};
}
